# vim: ts=2 sw=2 expandtab
import sys
from os import mkdir,listdir
from os.path import exists
import time
from urllib2 import urlopen
from xml.dom import Node
from xml.dom.minidom import parseString

debug=True

def getRemain():
  url='http://twitter.com/account/rate_limit_status.xml'
  raw=urlopen(url).read()
  dom=parseString(raw)
  tag=dom.getElementsByTagName('remaining-hits').item(0)
  remain=int(tag.firstChild.data)
  return remain

def getResetTime():
  url='http://twitter.com/account/rate_limit_status.xml'
  raw=urlopen(url).read()
  dom=parseString(raw)
  tag=dom.getElementsByTagName('reset-time').item(0)
  resetTime=tag.firstChild.data
  return resetTime

def getIds(rawdata):
  ids=[]
  dom=parseString(rawdata)
  for idNode in dom.getElementsByTagName('id'):
    ids.append(int(idNode.childNodes[0].data))
  return ids

def getPage(user, page):
  url='http://twitter.com/statuses/user_timeline/'
  rawData=urlopen(url+user+'.xml?page='+str(page)).read()
  return rawData

def debug(str):
  if debug: print 'DEBUG: '+str

### MAIN ####################################################
if(len(sys.argv)<2):
  print 'Usage: '+sys.argv[0]+' <twitter account name> [page offset]'
else:
  user=sys.argv[1]
  print 'twitter account name: '+user
  dir='twitter-log-'+user
  sleepTime=3
  
  # CONFIRM
  s=raw_input('Now logging start, take several times. r u OK? [y/N]: ')
  
  if s.lower()=='y' or s.lower()=='yes':
    # START LOGGING
    remain=getRemain()
    if(remain>0):
      idSet=set()                           # init
      if(not exists(dir)):
        mkdir(dir) # make dir
        debug('make dir: '+dir)
      else:
        debug('read previous files')
        for path in listdir(dir):
          debug('read %s' % (path))
          path=dir+'/'+path
          f=open(path, "r")
          prevlog=f.read()
          newIdSet=set(getIds(prevlog))
          idSet=idSet|newIdSet
          debug('%s items' % (len(idSet)))
      if(len(sys.argv)>2): page=int(sys.argv[2])
      else:                page=1
      pageStr='%04d'%page
      rawData=getPage(user, page)           # get log
      debug('get log: page='+str(page))
      newIdSet=set(getIds(rawData))-idSet
      idSet=idSet|newIdSet
      debug('new tweet: '+str(len(newIdSet))+', accum. tweet: '+str(len(idSet)))
      remain=getRemain()
      debug('remain hits: '+str(remain))
  
      while(len(newIdSet)>0 and remain>0):
        path=time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))+'-page'+pageStr+'.xml'
        f=open(dir+'/'+path, 'w')
        f.write(rawData)
        debug('write file: '+path)
  
        debug('wait '+str(sleepTime)+'sec.')
        time.sleep(sleepTime);                   # wait
  
        page+=1
        pageStr='%04d'%page
        rawData=getPage(user, page)            # get log
        debug('get log: page='+str(page))
        newIdSet=set(getIds(rawData))-idSet
        idSet=idSet|newIdSet
        debug('new tweet: '+str(len(newIdSet))+', accum. tweet: '+str(len(idSet)))
        remain=getRemain()
        debug('remain hits: '+str(remain))

      if(remain==0):
        print 'Error: no more request.'
        print 'Next reset time(UTC): '+getResetTime()
      else: print 'OK!'

    else:
      print 'Error: no more request.'
      print 'Next reset time(UTC): '+getResetTime()
  else: print 'Aborted.'
